{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.2"
    },
    "colab": {
      "name": "Albert.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "9b42748cf4124833bab47e6c7c76bafa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4f06d426558546e1b81930f60764a200",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_fd1aa0216ef649feb559b3cc9743a870",
              "IPY_MODEL_7b441cd29a4d4f78acda4d8ebc51ee91"
            ]
          }
        },
        "4f06d426558546e1b81930f60764a200": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fd1aa0216ef649feb559b3cc9743a870": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_837d0bd13d044e12b7fdeb30486273ca",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 238187,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 238187,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d5bce6ba5994482692e9ae43fbc2fd42"
          }
        },
        "7b441cd29a4d4f78acda4d8ebc51ee91": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_580b4716b09e4175a215e61fb2049986",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 238k/238k [00:00&lt;00:00, 689kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8c73d0e671ac4e6ea900a0032e4f7f51"
          }
        },
        "837d0bd13d044e12b7fdeb30486273ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d5bce6ba5994482692e9ae43fbc2fd42": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "580b4716b09e4175a215e61fb2049986": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8c73d0e671ac4e6ea900a0032e4f7f51": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a17a6f6014da4b3b9385ae431419bc37": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d9d465767e5f48fea25a349d144dbc8e",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f1af883ca7ec4131951335a42a8edb63",
              "IPY_MODEL_a6e478d25f1046a1b30272d4e1c787e9"
            ]
          }
        },
        "d9d465767e5f48fea25a349d144dbc8e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f1af883ca7ec4131951335a42a8edb63": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6a8150bee51d4cf68840a0e1bd4c98fc",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 505,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 505,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_eb4dee89c63d456983232fd187a0b70f"
          }
        },
        "a6e478d25f1046a1b30272d4e1c787e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c4262ad4902348cbbd2bc30c96bbfe9b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 505/505 [00:00&lt;00:00, 1.34kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_35868cdabf034b8eb1e78a0c86bc40c5"
          }
        },
        "6a8150bee51d4cf68840a0e1bd4c98fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "eb4dee89c63d456983232fd187a0b70f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c4262ad4902348cbbd2bc30c96bbfe9b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "35868cdabf034b8eb1e78a0c86bc40c5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "edb81e440e6f4e15a5068f80bb137447": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ca86e9295bfa4db58e334fbed8e30922",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8a208c9d197a4263befa6cdb9eb78de9",
              "IPY_MODEL_fc4e5eee921549d9b26efb3e7645158b"
            ]
          }
        },
        "ca86e9295bfa4db58e334fbed8e30922": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8a208c9d197a4263befa6cdb9eb78de9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_3f27ce4e4d094eea8891f0557fe59ff7",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 897396780,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 897396780,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b48f68f2a5ac4d7aa0843c3ec6691e39"
          }
        },
        "fc4e5eee921549d9b26efb3e7645158b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d69e073c60684d26a7b2d216489ca519",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 897M/897M [00:13&lt;00:00, 67.8MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_406dda830f69415880bb85ace83f73e2"
          }
        },
        "3f27ce4e4d094eea8891f0557fe59ff7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b48f68f2a5ac4d7aa0843c3ec6691e39": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d69e073c60684d26a7b2d216489ca519": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "406dda830f69415880bb85ace83f73e2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/agemagician/ProtTrans/blob/master/Embedding/PyTorch/Basic/ProtAlbert.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Vq0jRNAxjinX"
      },
      "source": [
        "<h3> Extracting protein sequences' features using ProtAlbert pretrained-model <h3>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ofDctgFKjinZ"
      },
      "source": [
        "<b>1. Load necessry libraries including huggingface transformers<b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "piYKiqH8jryM",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8d11658b-e8b2-4799-9f6c-762166fa2677"
      },
      "source": [
        "!pip install -q transformers sentencePiece"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 1.8MB 13.6MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.2MB 47.5MB/s \n",
            "\u001b[K     |████████████████████████████████| 890kB 44.0MB/s \n",
            "\u001b[K     |████████████████████████████████| 2.9MB 48.8MB/s \n",
            "\u001b[?25h  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "89O3T9mejina"
      },
      "source": [
        "import torch\n",
        "from transformers import AlbertTokenizer, AutoModel, pipeline\n",
        "import re\n",
        "import numpy as np\n",
        "import os\n",
        "import requests\n",
        "from tqdm.auto import tqdm"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8D7rhCPIjinn"
      },
      "source": [
        "<b>2. Load the vocabulary and ProtAlbert Model<b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F5bEAOWIjino",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "9b42748cf4124833bab47e6c7c76bafa",
            "4f06d426558546e1b81930f60764a200",
            "fd1aa0216ef649feb559b3cc9743a870",
            "7b441cd29a4d4f78acda4d8ebc51ee91",
            "837d0bd13d044e12b7fdeb30486273ca",
            "d5bce6ba5994482692e9ae43fbc2fd42",
            "580b4716b09e4175a215e61fb2049986",
            "8c73d0e671ac4e6ea900a0032e4f7f51"
          ]
        },
        "outputId": "e7d3660f-c74c-4e1f-a63a-4194b7f9ae6c"
      },
      "source": [
        "tokenizer = AlbertTokenizer.from_pretrained(\"Rostlab/prot_albert\", do_lower_case=False )"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9b42748cf4124833bab47e6c7c76bafa",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=238187.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jBS6buOpjioA",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 115,
          "referenced_widgets": [
            "a17a6f6014da4b3b9385ae431419bc37",
            "d9d465767e5f48fea25a349d144dbc8e",
            "f1af883ca7ec4131951335a42a8edb63",
            "a6e478d25f1046a1b30272d4e1c787e9",
            "6a8150bee51d4cf68840a0e1bd4c98fc",
            "eb4dee89c63d456983232fd187a0b70f",
            "c4262ad4902348cbbd2bc30c96bbfe9b",
            "35868cdabf034b8eb1e78a0c86bc40c5",
            "edb81e440e6f4e15a5068f80bb137447",
            "ca86e9295bfa4db58e334fbed8e30922",
            "8a208c9d197a4263befa6cdb9eb78de9",
            "fc4e5eee921549d9b26efb3e7645158b",
            "3f27ce4e4d094eea8891f0557fe59ff7",
            "b48f68f2a5ac4d7aa0843c3ec6691e39",
            "d69e073c60684d26a7b2d216489ca519",
            "406dda830f69415880bb85ace83f73e2"
          ]
        },
        "outputId": "dcdd0d30-aeac-43b8-de04-125c72c369e8"
      },
      "source": [
        "model = AutoModel.from_pretrained(\"Rostlab/prot_albert\")"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a17a6f6014da4b3b9385ae431419bc37",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=505.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "edb81e440e6f4e15a5068f80bb137447",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=897396780.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "osE9bx68jioE"
      },
      "source": [
        "<b>3. Load the model into the GPU if avilabile and switch to inference mode<b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aRUcC4LbjioF"
      },
      "source": [
        "fe = pipeline('feature-extraction', model=model, tokenizer=tokenizer,device=0)"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZljCGcZ3jioH"
      },
      "source": [
        "<b>4. Create or load sequences and map rarely occured amino acids (U,Z,O,B) to (X)<b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IMc4FBrUjioI"
      },
      "source": [
        "sequences_Example = [\"A E T C Z A O\",\"S K T Z P\"]"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HtCjAwfsjioM"
      },
      "source": [
        "sequences_Example = [re.sub(r\"[UZOB]\", \"X\", sequence) for sequence in sequences_Example]"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fCE2mLJZjioP"
      },
      "source": [
        "<b>5. Extracting sequences' features and covert the output to numpy if needed<b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_BaRfVLojioQ"
      },
      "source": [
        "embedding = fe(sequences_Example)"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GSU08Yc7jioT"
      },
      "source": [
        "embedding = np.array(embedding)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EAQ1lQxYjioZ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "47a0cbf5-b8b8-4361-d424-98f0ce3c6388"
      },
      "source": [
        "print(embedding)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[[ 0.05823962  0.01667558 -0.07323031 ...  0.0572243   0.05712279\n",
            "   -0.01743276]\n",
            "  [ 0.05041254  0.01670932 -0.06041706 ...  0.03879357  0.06743018\n",
            "   -0.02243913]\n",
            "  [ 0.04481506  0.02007634 -0.06041673 ...  0.01449918  0.07357343\n",
            "   -0.02993768]\n",
            "  ...\n",
            "  [ 0.03966146  0.04050564 -0.0189226  ...  0.02172126  0.06361968\n",
            "   -0.04445013]\n",
            "  [ 0.04467507  0.02889781 -0.06304034 ...  0.0083055   0.08290347\n",
            "   -0.02485393]\n",
            "  [ 0.05313296  0.0166304  -0.07059897 ...  0.04868817  0.06335299\n",
            "   -0.01911608]]\n",
            "\n",
            " [[-0.03029598  0.01223707 -0.02814652 ... -0.12239455 -0.11794612\n",
            "    0.06286692]\n",
            "  [-0.0316638   0.01354288 -0.02900524 ... -0.11965024 -0.11535397\n",
            "    0.0792664 ]\n",
            "  [-0.04238674  0.01891162 -0.02534152 ... -0.12102054 -0.10938807\n",
            "    0.09651565]\n",
            "  ...\n",
            "  [-0.03416523  0.01334407 -0.02930518 ... -0.12059835 -0.11478349\n",
            "    0.0759825 ]\n",
            "  [-0.04335058  0.02185565 -0.02340632 ... -0.12275703 -0.10738549\n",
            "    0.10495753]\n",
            "  [-0.04291698  0.02312784 -0.02060241 ... -0.12334881 -0.10543049\n",
            "    0.10805209]]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qeXTylW_jioq"
      },
      "source": [
        "<b>Optional: Remove padding ([PAD]) and special tokens ([CLS],[SEP]) that is added by ProtAlbert model<b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WiQ_lB9qjior"
      },
      "source": [
        "features = [] \n",
        "\n",
        "for seq_num in range(len(embedding)):\n",
        "    seq_len = len(sequences_Example[seq_num].replace(\" \", \"\"))\n",
        "    start_Idx = 1\n",
        "    end_Idx = seq_len+1\n",
        "    seq_emd = embedding[seq_num][start_Idx:end_Idx]\n",
        "    features.append(seq_emd)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L6Sg2i55jiot",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "29101fac-1fe3-4ae5-a1ae-851d0c94667a"
      },
      "source": [
        "print(features)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[array([[ 0.05041254,  0.01670932, -0.06041706, ...,  0.03879357,\n",
            "         0.06743018, -0.02243913],\n",
            "       [ 0.04481506,  0.02007634, -0.06041673, ...,  0.01449918,\n",
            "         0.07357343, -0.02993768],\n",
            "       [ 0.04433636,  0.02113897, -0.05988098, ...,  0.01485683,\n",
            "         0.07245776, -0.02932404],\n",
            "       ...,\n",
            "       [ 0.04479509,  0.01773552, -0.06211109, ...,  0.01833772,\n",
            "         0.07087235, -0.0290137 ],\n",
            "       [ 0.03966146,  0.04050564, -0.0189226 , ...,  0.02172126,\n",
            "         0.06361968, -0.04445013],\n",
            "       [ 0.04467507,  0.02889781, -0.06304034, ...,  0.0083055 ,\n",
            "         0.08290347, -0.02485393]]), array([[-0.0316638 ,  0.01354288, -0.02900524, ..., -0.11965024,\n",
            "        -0.11535397,  0.0792664 ],\n",
            "       [-0.04238674,  0.01891162, -0.02534152, ..., -0.12102054,\n",
            "        -0.10938807,  0.09651565],\n",
            "       [-0.04247037,  0.02128032, -0.02400274, ..., -0.12410947,\n",
            "        -0.10866702,  0.10307093],\n",
            "       [-0.04353855,  0.02157558, -0.02257719, ..., -0.12234777,\n",
            "        -0.10860761,  0.09839477],\n",
            "       [-0.04395629,  0.02232272, -0.02410841, ..., -0.12444437,\n",
            "        -0.10650399,  0.1127855 ]])]\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}